//////////////////////////////////////////////////////////////////////
//
//	Crytek CryENGINE Source code
//
//	File:ReadSequencerThread.cpp
//  Description: Loading file data ahead for fast load from DVD
//    Usually all CVars has to be set into release state for proper work of file sequencer:
//
//	History:
//	  -May 24,2010: Created by Vladimir Kajalin
//
//////////////////////////////////////////////////////////////////////

#include "StdAfx.h"
#include "CryPak.h"
#include <ILog.h>
#include <ISystem.h>
#include <IPlatformOS.h>
#include <IConsole.h>
#include <ITimer.h>
#include <IPerfHud.h>
#include "zlib/zlib.h"				// crc32()
#include "md5.h"

#if defined(PS3) || defined(LINUX)
#include "System.h"
#include <unistd.h>
#include <sys/stat.h>					// fstat, fileno
#endif





#include "ReadSequencerThread.h"

CReadSequencerThread::CReadSequencerThread(const char * szFileName)
{	
  m_bContinue=true;
  m_bDone=false;
  m_nLoadedStart=0;
  m_nLoadedEnd=0;
  m_nSeek=0;
  m_bEOF=false;

  m_nSeqBufferSize = gEnv->pConsole->GetCVar("sys_FileReadSequencerBufferSize")->GetIVal()*1024*1024;
  m_nSeqChunkSize = gEnv->pConsole->GetCVar("sys_FileReadSequencerBufferChunkSize")->GetIVal()*1024*1024;

  m_pBuffer = new byte[m_nSeqBufferSize];
  m_pBufferAsync = new byte[m_nSeqChunkSize];

  m_sFileName = szFileName;

  Start(0, "ReadSequencerThread");
}

CReadSequencerThread::~CReadSequencerThread()
{
  Stop();
  Cancel();
  WaitForThread();
  while(!m_bDone)
    CrySleep(10);
  Stop();
  WaitForThread();

  delete [] m_pBuffer;
  delete [] m_pBufferAsync;
}

int CReadSequencerThread::FreadLowLevel(void *pData, size_t nSize, size_t nCount, FILE * hFile)
{
  static CryCriticalSection csLowLevelRead;
  
  AUTO_LOCK(csLowLevelRead); 

  return ::fread(pData, nSize, nCount, hFile);
}

int CReadSequencerThread::Fread(void *pData, size_t nSize, size_t nCount, FILE * hFile)
{
  LOADING_TIME_PROFILE_SECTION;

  int nBytesToRead = nSize*nCount;

  int nNotReadyCount = 0;

  const int nSleep = 20;

  while(1)
  {
    if(m_bEOF && m_nSeek+nBytesToRead > m_nLoadedEnd)
    {
      AUTO_LOCK(m_csRead);

      if(m_bEOF && m_nSeek+nBytesToRead > m_nLoadedEnd)
      {
        m_nSeek = m_nLoadedEnd;
        return 0;
      }
    }

    if(m_nSeek >= m_nLoadedStart && m_nSeek+nBytesToRead <= m_nLoadedEnd)
    {
      AUTO_LOCK(m_csRead);

      if(m_nSeek >= m_nLoadedStart && m_nSeek+nBytesToRead <= m_nLoadedEnd)
      {
        if(nNotReadyCount)
          CFileReadSequencer::Log( "Data waiting stall: %d ms, %.1f MB, %d, %s ",            
          nNotReadyCount*nSleep, float(nBytesToRead)/1024.f/1024.f, m_nSeek, CFileReadSequencer::GetFileName(hFile) );

        if(pData)
          memcpy(pData, &m_pBuffer[m_nSeek-m_nLoadedStart], nBytesToRead);
        m_nSeek += nBytesToRead;

        return nCount;
      }
    }

    CrySleep(nSleep);

    nNotReadyCount++;

    if((nNotReadyCount%200)==0)
    {
      CFileReadSequencer::Log( "Data waiting ... : %d ms, %.1f MB, %d, %s ",            
        nNotReadyCount*nSleep, float(nBytesToRead)/1024.f/1024.f, m_nSeek, CFileReadSequencer::GetFileName(hFile) );
    }
  }
}

/*virtual void CReadSequencerThread::Run_()
{
CryThreadSetName( ~0, "ReadSequencerThread" );

FILE * pFile = ::fopen(m_sFileName,"rbS");

int nAsyncReady = 0;

int nReadChunkSize = m_nSeqChunkSize/256;

while(m_bContinue && pFile)
{       
if(!nAsyncReady)
nAsyncReady = FreadLowLevel(m_pBufferAsync, 1, nReadChunkSize, pFile);
else
CrySleep(10);

if((m_nSeek >= m_nLoadedStart+m_nSeqChunkSize || m_nLoadedEnd<m_nSeqBufferSize) && nAsyncReady)
{
AUTO_LOCK(m_csRead);

if((m_nSeek >= m_nLoadedStart+m_nSeqChunkSize || m_nLoadedEnd<m_nSeqBufferSize) && nAsyncReady)
{
if(m_nLoadedEnd<m_nSeqBufferSize)
{
// append new data
memcpy(&m_pBuffer[m_nLoadedEnd], m_pBufferAsync, nAsyncReady);        
m_nLoadedEnd += nAsyncReady;
}
else
{
// shift data
memcpy(&m_pBuffer[0],&m_pBuffer[m_nSeqChunkSize],m_nSeqBufferSize-m_nSeqChunkSize);

// append new data
memcpy(&m_pBuffer[m_nSeqBufferSize-m_nSeqChunkSize], m_pBufferAsync, nAsyncReady);        

// update loaded range
m_nLoadedStart += m_nSeqChunkSize;
m_nLoadedEnd += nAsyncReady;
}

// detect EOF
if(nAsyncReady != nReadChunkSize)
{
m_bEOF = true;
break;
}

if(m_nLoadedEnd >= m_nSeqChunkSize && nReadChunkSize != m_nSeqChunkSize)
{
//            CFileReadSequencer::Log( "ReadChunkSize is %d MB", 
//            m_nSeqChunkSize/1024/1024);

nReadChunkSize = m_nSeqChunkSize;
}

// request next read
nAsyncReady = 0;
}
}
}

m_bEOF = true;

if(pFile)
::fclose(pFile);

m_bDone = true;
}*/

void CReadSequencerThread::Run()
{
  CryThreadSetName( ~0, "ReadSequencerThread" );

  FILE * pFile = ::fopen(m_sFileName.c_str(),"rbS");

  if(!pFile)
    CFileReadSequencer::Log( "Error opening %s", m_sFileName.c_str() );

  FILE * pRefFile = 0;
  char szOpenRefFileName[256]="";
  int nCurRefFileSeek = 0;

  int nAsyncReady = 0;

  int nCycles0=0, nCycles1=0, nCycles2=0, nCycles3=0;

  while(m_bContinue && pFile)
  {       
    if(!nAsyncReady)
    {
      SSeqHeader & seqHeader = *(SSeqHeader *)m_pBufferAsync;
      nAsyncReady += FreadLowLevel(&seqHeader, 1, sizeof(seqHeader), pFile);
      if(nAsyncReady != sizeof(seqHeader))
      {
        break;
      }

      if(strncmp(seqHeader.sMarker, "CRY", sizeof(seqHeader.sMarker)))
      {
        CFileReadSequencer::Log( "ERROR0: CRY" );
        break;
      }

      if((int)(seqHeader.nSize*seqHeader.nItemsRead+sizeof(seqHeader)) > m_nSeqChunkSize)
      {
        CFileReadSequencer::Log( "ERROR1: block size error: %d MB", (seqHeader.nSize*seqHeader.nItemsRead+sizeof(seqHeader))/1024/1024 );
        break;
      }

      if(seqHeader.nBlockGroupSize>m_nSeqChunkSize)
      {
        CFileReadSequencer::Log( "ERROR2: size error: (seqHeader.nBlockGroupSize=%d)>m_nSeqChunkSize", seqHeader.nBlockGroupSize/1024/1024);
        break;
      }

      if(seqHeader.nFileNameLen>0)
      { // dependent read
        nCycles0++;
        char szFileName[256]="";
        FreadLowLevel(szFileName, 1, seqHeader.nFileNameLen, pFile);
        szFileName[seqHeader.nFileNameLen]=0;

        if(!pRefFile || strcmp(szOpenRefFileName,szFileName) != 0)
        {
          if(pRefFile)
            ::fclose(pRefFile);

          pRefFile = ::fopen(szFileName, "rbS");
          if(!pRefFile)
          {
            CFileReadSequencer::Log( "ERROR3: referenced fopen error" );
            break;
          }
          strcpy(szOpenRefFileName,szFileName);
          nCurRefFileSeek = 0;

          //          CFileReadSequencer::Log("%d - %s", seqHeader.nCount*seqHeader.nSize, szFileName);
        }

        if(seqHeader.nSeek != nCurRefFileSeek)
        {
          ::fseek(pRefFile, seqHeader.nSeek, SEEK_SET);
          nCurRefFileSeek = seqHeader.nSeek;
        }

        nAsyncReady += FreadLowLevel(m_pBufferAsync+sizeof(seqHeader), 1, seqHeader.nSize*seqHeader.nItemsRead, pRefFile);
        if(nAsyncReady != (seqHeader.nSize*seqHeader.nItemsRead+sizeof(seqHeader)))
        {
          CFileReadSequencer::Log( "ERROR3: referenced data read error" );
          break;
        }
        nCurRefFileSeek += seqHeader.nSize*seqHeader.nItemsRead;
      }
      else if(seqHeader.nBlockGroupSize>0)
      { // large block read
        nCycles1++;
        nAsyncReady += FreadLowLevel(m_pBufferAsync+sizeof(seqHeader), 1, seqHeader.nBlockGroupSize-sizeof(seqHeader), pFile);
        if(nAsyncReady != (seqHeader.nBlockGroupSize))
        {
          CFileReadSequencer::Log( "ERROR3: data read error" );
          break;
        }
      }
      else if(seqHeader.nBlockGroupSize<0)
      { // read all till the end
        float fStartTime = gEnv->pTimer->GetAsyncCurTime();          
        nCycles2++;
        nAsyncReady += FreadLowLevel(m_pBufferAsync+sizeof(seqHeader), 1, m_nSeqChunkSize-sizeof(seqHeader), pFile);
        CFileReadSequencer::Log( "Read %d MB till the end: %.1f sec",
          nAsyncReady/1024/1024, max(0.f, gEnv->pTimer->GetAsyncCurTime() - fStartTime));
      }
      else
      { // just read single block
        nCycles3++;
        nAsyncReady += FreadLowLevel(m_pBufferAsync+sizeof(seqHeader), 1, seqHeader.nSize*seqHeader.nItemsRead, pFile);
        if(nAsyncReady != (seqHeader.nSize*seqHeader.nItemsRead+sizeof(seqHeader)))
        {
          CFileReadSequencer::Log( "ERROR4: data read error" );
          break;
        }
      }
    }
    else
      CrySleep(10);

    // shift
    int nShiftThreshold = 1024*1024;
    int nMemWasted = m_nSeek-m_nLoadedStart;

    if(nMemWasted >= nShiftThreshold)
    {
      AUTO_LOCK(m_csRead);

      if(nMemWasted >= nShiftThreshold)
      {
        // shift data
        memcpy(&m_pBuffer[0],&m_pBuffer[nMemWasted],m_nSeqBufferSize-nMemWasted);
        m_nLoadedStart += nMemWasted;
      }
    }

    // append
    if(nAsyncReady && (((m_nLoadedEnd-m_nLoadedStart)+nAsyncReady)<=m_nSeqBufferSize))
    {
      AUTO_LOCK(m_csRead);

      if(nAsyncReady && (((m_nLoadedEnd-m_nLoadedStart)+nAsyncReady)<=m_nSeqBufferSize))
      {
        // append new data
        assert((m_nLoadedEnd-m_nLoadedStart)<=m_nSeqBufferSize);

        memcpy(&m_pBuffer[m_nLoadedEnd-m_nLoadedStart], m_pBufferAsync, nAsyncReady);        
        m_nLoadedEnd += nAsyncReady;
        nAsyncReady = 0;
      }
    }
  }

  m_bEOF = true;

  if(pFile)
    ::fclose(pFile);

  if(pRefFile)
    ::fclose(pRefFile);

  m_bDone = true;

  CFileReadSequencer::Log( "Finished reading sequencer file: Cycles: %d / %d / %d / %d",            
    nCycles0, nCycles1, nCycles2, nCycles3 );
}
